{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n",
    "\n",
    "**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n",
    "\n",
    "This notebook was generated for TensorFlow 2.6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# The mathematical building blocks of neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## A first look at a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Loading the MNIST dataset in Keras**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60000"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([7, 2, 1, ..., 4, 5, 6], dtype=uint8)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**The network architecture**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**The compilation step**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"sparse_categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Preparing the image data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**\"Fitting\" the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 3ms/step - accuracy: 0.8737 - loss: 0.4427\n",
      "Epoch 2/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9650 - loss: 0.1198\n",
      "Epoch 3/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9779 - loss: 0.0723\n",
      "Epoch 4/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9856 - loss: 0.0496\n",
      "Epoch 5/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9886 - loss: 0.0378\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x1991a7b6ba0>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Using the model to make predictions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 56ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([6.1386096e-08, 4.0220733e-09, 1.4019292e-05, 1.0912131e-04,\n",
       "       1.6050739e-11, 2.0000472e-08, 2.1586470e-12, 9.9987638e-01,\n",
       "       3.5246282e-07, 1.0555324e-07], dtype=float32)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digits = test_images[0:10]\n",
    "predictions = model.predict(test_digits)\n",
    "predictions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.int64(7)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0].argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float32(0.9998764)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0][7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.uint8(7)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Evaluating the model on new data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step - accuracy: 0.9735 - loss: 0.0875\n",
      "test_acc: 0.9776999950408936\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(test_images, test_labels)\n",
    "print(f\"test_acc: {test_acc}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Data representations for neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Scalars (rank-0 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(12)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "x = np.array(12)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Vectors (rank-1 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12,  3,  6, 14,  7])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([12, 3, 6, 14, 7])\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Matrices (rank-2 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[5, 78, 2, 34, 0],\n",
    "              [6, 79, 3, 35, 1],\n",
    "              [7, 80, 4, 36, 2]])\n",
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Rank-3 and higher-rank tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]],\n",
    "              [[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]],\n",
    "              [[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]]])\n",
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Key attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('uint8')"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Displaying the fourth digit**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAGV5JREFUeJzt3XuMFdXhB/CzqKyIsBQRlpUFwfcDaeqDEl9YCCtNjShppdoGGgKRghGp1WBV1Dbdn5pao0X8p4XaqlATgegfWEXZrS3YgiWUPqhLqGB4arLLQwEL88uM2S2rIN7L7p7dez+f5OTuvXfOzmGYne89M2fOLUmSJAkA0MY6tfUKASAlgACIQgABEIUAAiAKAQRAFAIIgCgEEABRCCAAojg+tDMHDx4MmzdvDt26dQslJSWxmwNAjtL5DXbt2hUqKipCp06dOk4ApeFTWVkZuxkAHKNNmzaFfv36dZwASns+jQ3v3r177OYAkKOdO3dmHYnG43mbB9Ds2bPDo48+GrZu3RqGDBkSnnzyyXDZZZcdtV7jabc0fAQQQMd1tMsorTIIYcGCBWHGjBlh1qxZ4e23384CqKqqKmzfvr01VgdAB9QqAfTYY4+FSZMmhe9973vh/PPPD08//XQ46aSTwq9+9avWWB0AHVCLB9D+/fvDqlWrwsiRI/+3kk6dsufLly//zPL79u3LzhceWgAofC0eQO+//344cOBA6NOnT7PX0+fp9aBPq66uDmVlZU3FCDiA4hD9RtSZM2eGhoaGppKOfgOg8LX4KLhevXqF4447Lmzbtq3Z6+nz8vLyzyxfWlqaFQCKS4v3gDp37hwuvvjisHTp0mazG6TPhw0b1tKrA6CDapX7gNIh2OPHjw+XXHJJdu/P448/Hvbs2ZONigOAVgugm266KezYsSPcf//92cCDL3/5y2HJkiWfGZgAQPEqSdJZ49qRdBh2OhouHZBgJgSAjueLHsejj4IDoDgJIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEBhBNADDzwQSkpKmpVzzz23pVcDQAd3fGv80gsuuCC89tpr/1vJ8a2yGgA6sFZJhjRwysvLW+NXA1AgWuUa0DvvvBMqKirCoEGDwi233BI2btx4xGX37dsXdu7c2awAUPhaPICGDh0a5s2bF5YsWRLmzJkTNmzYEK688sqwa9euwy5fXV0dysrKmkplZWVLNwmAdqgkSZKkNVdQX18fBgwYEB577LEwceLEw/aA0tIo7QGlIdTQ0BC6d+/emk0DoBWkx/G0Q3G043irjw7o0aNHOPvss0NdXd1h3y8tLc0KAMWl1e8D2r17d1i/fn3o27dva68KgGIOoDvvvDPU1NSE//znP+FPf/pTuOGGG8Jxxx0Xvv3tb7f0qgDowFr8FNx7772Xhc0HH3wQTj311HDFFVeEFStWZD8DQKsF0Pz581v6VwJQgMwFB0AUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiaPUvpIOO5K233sq5zm9+85uc69TW1uZcZ+3ataGt/OxnP8u5TkVFRc51/vCHP+Rc57vf/W7OdYYOHZpzHVqfHhAAUQggAKIQQABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEIYAAiEIAARCF2bApSAsWLMir3u23355znR07duRcJ0mSnOsMHz485zrvv/9+yMedd94Z2kI+2yGff9P8+fNzrkPr0wMCIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFGYjJQ29d///jfnOn/5y19yrjNp0qSQjz179uRc5+qrr865zn333ZdznSuuuCLnOvv27Qv5+Na3vpVznVdeeSW0hUsuuaRN1kPr0wMCIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFGYjJQ29dvf/jbnOhMnTgxtZdSoUTnXWbBgQc51unfvHtpCPm1ry4lFKysrc64zfvz4VmkLbU8PCIAoBBAAHSOAamtrw3XXXRcqKipCSUlJWLRoUbP3kyQJ999/f+jbt2/o0qVLGDlyZHjnnXdass0AFGMApV/YNWTIkDB79uzDvv/II4+EJ554Ijz99NPhrbfeCl27dg1VVVVh7969LdFeAIp1EMLo0aOzcjhp7+fxxx8P9957b7j++uuz15555pnQp0+frKc0bty4Y28xAAWhRa8BbdiwIWzdujU77daorKwsDB06NCxfvvyIXxm8c+fOZgWAwteiAZSGTyrt8Rwqfd743qdVV1dnIdVY8hmWCUDHE30U3MyZM0NDQ0NT2bRpU+wmAdDRAqi8vDx73LZtW7PX0+eN731aaWlpdlPeoQWAwteiATRw4MAsaJYuXdr0WnpNJx0NN2zYsJZcFQDFNgpu9+7doa6urtnAg9WrV4eePXuG/v37h+nTp4ef/OQn4ayzzsoC6b777svuGRozZkxLtx2AYgqglStXhmuuuabp+YwZM5rmZ5o3b1646667snuFJk+eHOrr68MVV1wRlixZEk488cSWbTkAHVpJkt68046kp+zS0XDpgATXg9q39H6vXP30pz/NuU4640aupk6dGvKR9t5z1Z730/POOy+vev/+979DW3jxxRdzrtN4jyHt1xc9jkcfBQdAcRJAAEQhgACIQgABEIUAAiAKAQRAFAIIgCgEEABRCCAAohBAAEQhgACIQgABEIUAAqBjfB0Dheehhx7Kq14+M1un34Cbq6qqqpzrPPzwwyEfXbp0CW1h7969Odf5/e9/n3Odd999N+Qjn0ny0+/+ypWZrYubHhAAUQggAKIQQABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEIYAAiMJkpAWmvr4+5zpPPfVUXusqKSlpk4lFFy1aFNqzurq6nOvccsstOddZuXJlaCvf/OY3c65z1113tUpbKFx6QABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEIYAAiEIAARCFAAIgCpORFpj9+/fnXGfHjh2hrTzxxBM519m+fXvOdebOnRvysXjx4pzr/P3vf8+5zq5du9pk8tdOnfL7jPmd73wn5zpdu3bNa10ULz0gAKIQQABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEIYAAiEIAARCFyUgLTOfOnXOu07t377zWlc8koaeffnqbTMLZlk477bSc63Tv3j3nOps3b865Tq9evUI+rrvuurzqQS70gACIQgAB0DECqLa2NuueV1RUZKdGFi1a1Oz9CRMmZK8fWq699tqWbDMAxRhAe/bsCUOGDAmzZ88+4jJp4GzZsqWpPP/888faTgCKfRDC6NGjs/J5SktLQ3l5+bG0C4AC1yrXgJYtW5aNrDrnnHPClClTwgcffHDEZfft2xd27tzZrABQ+Fo8gNLTb88880xYunRpePjhh0NNTU3WYzpw4MBhl6+urg5lZWVNpbKysqWbBEAx3Ac0bty4pp8HDx4cLrroonDGGWdkvaIRI0Z8ZvmZM2eGGTNmND1Pe0BCCKDwtfow7EGDBmU3w9XV1R3xelF6U96hBYDC1+oB9N5772XXgPr27dvaqwKgkE/B7d69u1lvZsOGDWH16tWhZ8+eWXnwwQfD2LFjs1Fw69evD3fddVc488wzQ1VVVUu3HYBiCqCVK1eGa665pul54/Wb8ePHhzlz5oQ1a9aEX//616G+vj67WXXUqFHhxz/+cXaqDQDyDqDhw4eHJEmO+P4rr7yS66+kBfXo0SPnOp+ezeKL+sY3vpFznc8bkn8kaQ86V9dff33IRzqTR67Snv+xDNZpzclI81kPtBVzwQEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAIXxldx0PEOHDs2r3o4dO1q8LR1RbW1tznVqampyrlNSUpLXNxJDe6UHBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiMBkpHKOPPvqoTSYWzafOuHHjcq4DbUUPCIAoBBAAUQggAKIQQABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEYTJSOEZVVVWxmwAdkh4QAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIjCZKRwjF555ZXYTYAOSQ8IgCgEEADtP4Cqq6vDpZdeGrp16xZ69+4dxowZE9atW9dsmb1794apU6eGU045JZx88slh7NixYdu2bS3dbgCKKYBqamqycFmxYkV49dVXw8cffxxGjRoV9uzZ07TMHXfcEV566aXwwgsvZMtv3rw53Hjjja3RdgCKZRDCkiVLmj2fN29e1hNatWpVuOqqq0JDQ0P45S9/GZ577rnwta99LVtm7ty54bzzzstC66tf/WrLth6A4rwGlAZOqmfPntljGkRpr2jkyJFNy5x77rmhf//+Yfny5Yf9Hfv27Qs7d+5sVgAofHkH0MGDB8P06dPD5ZdfHi688MLsta1bt4bOnTuHHj16NFu2T58+2XtHuq5UVlbWVCorK/NtEgDFEEDptaC1a9eG+fPnH1MDZs6cmfWkGsumTZuO6fcBUMA3ok6bNi28/PLLoba2NvTr16/p9fLy8rB///5QX1/frBeUjoJL3zuc0tLSrABQXHLqASVJkoXPwoULw+uvvx4GDhzY7P2LL744nHDCCWHp0qVNr6XDtDdu3BiGDRvWcq0GoLh6QOlpt3SE2+LFi7N7gRqv66TXbrp06ZI9Tpw4McyYMSMbmNC9e/dw2223ZeFjBBwAeQfQnDlzssfhw4c3ez0daj1hwoTs55///OehU6dO2Q2o6Qi3qqqq8NRTT+WyGgCKwPG5noI7mhNPPDHMnj07K1AM1q9fH7sJ0CGZCw6AKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAOg434gK/M+VV16Zc50vMrM8FDo9IACiEEAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhclI4RgNHjw45zpnnXVWznXWr1/fJnVSp556al71IBd6QABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEIYAAiEIAARCFAAIgCpORQgT33HNPznUmTpzYJutJ/eIXv8i5zvnnn5/XuiheekAARCGAAIhCAAEQhQACIAoBBEAUAgiAKAQQAFEIIACiEEAARCGAAIhCAAEQhQACIAqTkUIEN954Y8515s+fn3OdV199NeTjgQceyLnO3Llzc67TtWvXnOtQOPSAAIhCAAHQ/gOouro6XHrppaFbt26hd+/eYcyYMWHdunXNlhk+fHgoKSlpVm699daWbjcAxRRANTU1YerUqWHFihXZueWPP/44jBo1KuzZs6fZcpMmTQpbtmxpKo888khLtxuAYhqEsGTJkmbP582bl/WEVq1aFa666qqm10866aRQXl7ecq0EoOAc0zWghoaG7LFnz57NXn/22WdDr169woUXXhhmzpwZPvzwwyP+jn379oWdO3c2KwAUvryHYR88eDBMnz49XH755VnQNLr55pvDgAEDQkVFRVizZk24++67s+tEL7744hGvKz344IP5NgOAYgug9FrQ2rVrw5tvvtns9cmTJzf9PHjw4NC3b98wYsSIsH79+nDGGWd85vekPaQZM2Y0PU97QJWVlfk2C4BCDqBp06aFl19+OdTW1oZ+/fp97rJDhw7NHuvq6g4bQKWlpVkBoLjkFEBJkoTbbrstLFy4MCxbtiwMHDjwqHVWr16dPaY9IQDIK4DS027PPfdcWLx4cXYv0NatW7PXy8rKQpcuXbLTbOn7X//618Mpp5ySXQO64447shFyF110US6rAqDA5RRAc+bMabrZ9NNzQE2YMCF07tw5vPbaa+Hxxx/P7g1Kr+WMHTs23HvvvS3bagCK7xTc50kDJ71ZFQCOpiQ5Wqq0sXQUXHpKL73HqHv37rGbA+1GPvfI/ehHP8prXU899VTOdf72t7/lXOf888/PuQ7t3xc9jpuMFIAoBBAAUQggAKIQQABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEYTJSAFqUyUgBaNcEEABRCCAAohBAAEQhgACIQgABEIUAAiAKAQRAFAIIgCgEEABRCCAAojg+tDONU9OlcwkB0PE0Hr+PNtVouwugXbt2ZY+VlZWxmwLAMR7P00lJO8xs2AcPHgybN28O3bp1CyUlJZ9J1TSYNm3aVNQzZdsOn7AdPmE7fMJ2aD/bIY2VNHwqKipCp06dOk4PKG1sv379PneZdKMW8w7WyHb4hO3wCdvhE7ZD+9gOn9fzaWQQAgBRCCAAouhQAVRaWhpmzZqVPRYz2+ETtsMnbIdP2A4dbzu0u0EIABSHDtUDAqBwCCAAohBAAEQhgACIosME0OzZs8Ppp58eTjzxxDB06NDw5z//ORSbBx54IJsd4tBy7rnnhkJXW1sbrrvuuuyu6vTfvGjRombvp+No7r///tC3b9/QpUuXMHLkyPDOO++EYtsOEyZM+Mz+ce2114ZCUl1dHS699NJsppTevXuHMWPGhHXr1jVbZu/evWHq1KnhlFNOCSeffHIYO3Zs2LZtWyi27TB8+PDP7A+33npraE86RAAtWLAgzJgxIxta+Pbbb4chQ4aEqqqqsH379lBsLrjggrBly5am8uabb4ZCt2fPnuz/PP0QcjiPPPJIeOKJJ8LTTz8d3nrrrdC1a9ds/0gPRMW0HVJp4By6fzz//POhkNTU1GThsmLFivDqq6+Gjz/+OIwaNSrbNo3uuOOO8NJLL4UXXnghWz6d2uvGG28MxbYdUpMmTWq2P6R/K+1K0gFcdtllydSpU5ueHzhwIKmoqEiqq6uTYjJr1qxkyJAhSTFLd9mFCxc2PT948GBSXl6ePProo02v1dfXJ6Wlpcnzzz+fFMt2SI0fPz65/vrrk2Kyffv2bFvU1NQ0/d+fcMIJyQsvvNC0zD//+c9smeXLlyfFsh1SV199dXL77bcn7Vm77wHt378/rFq1Kjutcuh8cenz5cuXh2KTnlpKT8EMGjQo3HLLLWHjxo2hmG3YsCFs3bq12f6RzkGVnqYtxv1j2bJl2SmZc845J0yZMiV88MEHoZA1NDRkjz179swe02NF2hs4dH9IT1P379+/oPeHhk9th0bPPvts6NWrV7jwwgvDzJkzw4cffhjak3Y3Gemnvf/+++HAgQOhT58+zV5Pn//rX/8KxSQ9qM6bNy87uKTd6QcffDBceeWVYe3atdm54GKUhk/qcPtH43vFIj39lp5qGjhwYFi/fn245557wujRo7MD73HHHRcKTTpz/vTp08Pll1+eHWBT6f95586dQ48ePYpmfzh4mO2Quvnmm8OAAQOyD6xr1qwJd999d3ad6MUXXwztRbsPIP4nPZg0uuiii7JASnew3/3ud2HixIlR20Z848aNa/p58ODB2T5yxhlnZL2iESNGhEKTXgNJP3wVw3XQfLbD5MmTm+0P6SCddD9IP5yk+0V70O5PwaXdx/TT26dHsaTPy8vLQzFLP+WdffbZoa6uLhSrxn3A/vFZ6Wna9O+nEPePadOmhZdffjm88cYbzb6+Jf0/T0/b19fXF8X+MO0I2+Fw0g+sqfa0P7T7AEq70xdffHFYunRpsy5n+nzYsGGhmO3evTv7NJN+silW6emm9MBy6P6RfiFXOhqu2PeP9957L7sGVEj7Rzr+Ij3oLly4MLz++uvZ//+h0mPFCSec0Gx/SE87pddKC2l/SI6yHQ5n9erV2WO72h+SDmD+/PnZqKZ58+Yl//jHP5LJkycnPXr0SLZu3ZoUkx/84AfJsmXLkg0bNiR//OMfk5EjRya9evXKRsAUsl27diV//etfs5Luso899lj287vvvpu9/3//93/Z/rB48eJkzZo12UiwgQMHJh999FFSLNshfe/OO+/MRnql+8drr72WfOUrX0nOOuusZO/evUmhmDJlSlJWVpb9HWzZsqWpfPjhh03L3HrrrUn//v2T119/PVm5cmUybNiwrBSSKUfZDnV1dclDDz2U/fvT/SH92xg0aFBy1VVXJe1Jhwig1JNPPpntVJ07d86GZa9YsSIpNjfddFPSt2/fbBucdtpp2fN0Ryt0b7zxRnbA/XRJhx03DsW+7777kj59+mQfVEaMGJGsW7cuKabtkB54Ro0alZx66qnZMOQBAwYkkyZNKrgPaYf796dl7ty5TcukHzy+//3vJ1/60peSk046Kbnhhhuyg3MxbYeNGzdmYdOzZ8/sb+LMM89MfvjDHyYNDQ1Je+LrGACIot1fAwKgMAkgAKIQQABEIYAAiEIAARCFAAIgCgEEQBQCCIAoBBAAUQggAKIQQABEIYAACDH8PwdKtRkWRcXEAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "digit = train_images[4]\n",
    "plt.imshow(digit, cmap=plt.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.uint8(9)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels[4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Manipulating tensors in NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100, :, :]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100, 0:28, 0:28]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_slice = train_images[:, 14:, 14:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_slice = train_images[:, 7:-7, 7:-7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### The notion of data batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch = train_images[:128]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch = train_images[128:256]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "n = 3\n",
    "batch = train_images[128 * n:128 * (n + 1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Real-world examples of data tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Vector data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Timeseries data or sequence data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Image data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Video data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The gears of neural networks: tensor operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Element-wise operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_relu(x):\n",
    "    assert len(x.shape) == 2\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] = max(x[i, j], 0)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_add(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert x.shape == y.shape\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] += y[i, j]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 0.00 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "x = np.random.random((20, 100))\n",
    "y = np.random.random((20, 100))\n",
    "\n",
    "t0 = time.time()\n",
    "for _ in range(1000):\n",
    "    z = x + y\n",
    "    z = np.maximum(z, 0.)\n",
    "print(\"Took: {0:.2f} s\".format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 0.99 s\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "for _ in range(1000):\n",
    "    z = naive_add(x, y)\n",
    "    z = naive_relu(z)\n",
    "print(\"Took: {0:.2f} s\".format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Broadcasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "X = np.random.random((32, 10))\n",
    "y = np.random.random((10,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "y = np.expand_dims(y, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "Y = np.concatenate([y] * 32, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_add_matrix_and_vector(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] += y[j]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "x = np.random.random((64, 3, 32, 10))\n",
    "y = np.random.random((32, 10))\n",
    "z = np.maximum(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Tensor product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = np.random.random((32,))\n",
    "y = np.random.random((32,))\n",
    "z = np.dot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_vector_dot(x, y):\n",
    "    assert len(x.shape) == 1\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[0] == y.shape[0]\n",
    "    z = 0.\n",
    "    for i in range(x.shape[0]):\n",
    "        z += x[i] * y[i]\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_vector_dot(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    z = np.zeros(x.shape[0])\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            z[i] += x[i, j] * y[j]\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_vector_dot(x, y):\n",
    "    z = np.zeros(x.shape[0])\n",
    "    for i in range(x.shape[0]):\n",
    "        z[i] = naive_vector_dot(x[i, :], y)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_dot(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 2\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    z = np.zeros((x.shape[0], y.shape[1]))\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(y.shape[1]):\n",
    "            row_x = x[i, :]\n",
    "            column_y = y[:, j]\n",
    "            z[i, j] = naive_vector_dot(row_x, column_y)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Tensor reshaping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.reshape((60000, 28 * 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 2)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[0., 1.],\n",
    "             [2., 3.],\n",
    "             [4., 5.]])\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.],\n",
       "       [1.],\n",
       "       [2.],\n",
       "       [3.],\n",
       "       [4.],\n",
       "       [5.]])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = x.reshape((6, 1))\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 300)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.zeros((300, 20))\n",
    "x = np.transpose(x)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Geometric interpretation of tensor operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A geometric interpretation of deep learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The engine of neural networks: gradient-based optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### What's a derivative?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Derivative of a tensor operation: the gradient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Stochastic gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Chaining derivatives: The Backpropagation algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The chain rule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Automatic differentiation with computation graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The gradient tape in TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "x = tf.Variable(0.)\n",
    "with tf.GradientTape() as tape:\n",
    "    y = 2 * x + 3\n",
    "grad_of_y_wrt_x = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = tf.Variable(tf.random.uniform((2, 2)))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = 2 * x + 3\n",
    "grad_of_y_wrt_x = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.random.uniform((2, 2)))\n",
    "b = tf.Variable(tf.zeros((2,)))\n",
    "x = tf.random.uniform((2, 2))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = tf.matmul(x, W) + b\n",
    "grad_of_y_wrt_W_and_b = tape.gradient(y, [W, b])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Looking back at our first example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"sparse_categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.8727 - loss: 0.4431\n",
      "Epoch 2/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9657 - loss: 0.1217\n",
      "Epoch 3/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 5ms/step - accuracy: 0.9783 - loss: 0.0745\n",
      "Epoch 4/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9853 - loss: 0.0488\n",
      "Epoch 5/5\n",
      "\u001b[1m469/469\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 4ms/step - accuracy: 0.9890 - loss: 0.0380\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x19920f61a60>"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Reimplementing our first example from scratch in TensorFlow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A simple Dense class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class NaiveDense:\n",
    "    def __init__(self, input_size, output_size, activation):\n",
    "        self.activation = activation\n",
    "\n",
    "        w_shape = (input_size, output_size)\n",
    "        w_initial_value = tf.random.uniform(w_shape, minval=0, maxval=1e-1)\n",
    "        self.W = tf.Variable(w_initial_value)\n",
    "\n",
    "        b_shape = (output_size,)\n",
    "        b_initial_value = tf.zeros(b_shape)\n",
    "        self.b = tf.Variable(b_initial_value)\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        return self.activation(tf.matmul(inputs, self.W) + self.b)\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "        return [self.W, self.b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A simple Sequential class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "class NaiveSequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        x = inputs\n",
    "        for layer in self.layers:\n",
    "           x = layer(x)\n",
    "        return x\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "       weights = []\n",
    "       for layer in self.layers:\n",
    "           weights += layer.weights\n",
    "       return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = NaiveSequential([\n",
    "    NaiveDense(input_size=28 * 28, output_size=512, activation=tf.nn.relu),\n",
    "    NaiveDense(input_size=512, output_size=10, activation=tf.nn.softmax)\n",
    "])\n",
    "assert len(model.weights) == 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A batch generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class BatchGenerator:\n",
    "    def __init__(self, images, labels, batch_size=128):\n",
    "        assert len(images) == len(labels)\n",
    "        self.index = 0\n",
    "        self.images = images\n",
    "        self.labels = labels\n",
    "        self.batch_size = batch_size\n",
    "        self.num_batches = math.ceil(len(images) / batch_size)\n",
    "\n",
    "    def next(self):\n",
    "        images = self.images[self.index : self.index + self.batch_size]\n",
    "        labels = self.labels[self.index : self.index + self.batch_size]\n",
    "        self.index += self.batch_size\n",
    "        return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Running one training step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def one_training_step(model, images_batch, labels_batch):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(images_batch)\n",
    "        per_sample_losses = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "            labels_batch, predictions)\n",
    "        average_loss = tf.reduce_mean(per_sample_losses)\n",
    "    gradients = tape.gradient(average_loss, model.weights)\n",
    "    update_weights(gradients, model.weights)\n",
    "    return average_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "learning_rate = 1e-3\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    for g, w in zip(gradients, weights):\n",
    "        w.assign_sub(g * learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "\n",
    "optimizer = optimizers.SGD(learning_rate=1e-3)\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    optimizer.apply_gradients(zip(gradients, weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### The full training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def fit(model, images, labels, epochs, batch_size=128):\n",
    "    for epoch_counter in range(epochs):\n",
    "        print(f\"Epoch {epoch_counter}\")\n",
    "        batch_generator = BatchGenerator(images, labels)\n",
    "        for batch_counter in range(batch_generator.num_batches):\n",
    "            images_batch, labels_batch = batch_generator.next()\n",
    "            loss = one_training_step(model, images_batch, labels_batch)\n",
    "            if batch_counter % 100 == 0:\n",
    "                print(f\"loss at batch {batch_counter}: {loss:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "loss at batch 0: 5.27\n",
      "loss at batch 100: 2.22\n",
      "loss at batch 200: 2.22\n",
      "loss at batch 300: 2.10\n",
      "loss at batch 400: 2.23\n",
      "Epoch 1\n",
      "loss at batch 0: 1.93\n",
      "loss at batch 100: 1.87\n",
      "loss at batch 200: 1.84\n",
      "loss at batch 300: 1.73\n",
      "loss at batch 400: 1.84\n",
      "Epoch 2\n",
      "loss at batch 0: 1.61\n",
      "loss at batch 100: 1.57\n",
      "loss at batch 200: 1.51\n",
      "loss at batch 300: 1.44\n",
      "loss at batch 400: 1.53\n",
      "Epoch 3\n",
      "loss at batch 0: 1.35\n",
      "loss at batch 100: 1.34\n",
      "loss at batch 200: 1.25\n",
      "loss at batch 300: 1.22\n",
      "loss at batch 400: 1.29\n",
      "Epoch 4\n",
      "loss at batch 0: 1.15\n",
      "loss at batch 100: 1.16\n",
      "loss at batch 200: 1.05\n",
      "loss at batch 300: 1.06\n",
      "loss at batch 400: 1.12\n",
      "Epoch 5\n",
      "loss at batch 0: 1.00\n",
      "loss at batch 100: 1.02\n",
      "loss at batch 200: 0.91\n",
      "loss at batch 300: 0.94\n",
      "loss at batch 400: 1.00\n",
      "Epoch 6\n",
      "loss at batch 0: 0.89\n",
      "loss at batch 100: 0.91\n",
      "loss at batch 200: 0.80\n",
      "loss at batch 300: 0.85\n",
      "loss at batch 400: 0.91\n",
      "Epoch 7\n",
      "loss at batch 0: 0.80\n",
      "loss at batch 100: 0.83\n",
      "loss at batch 200: 0.72\n",
      "loss at batch 300: 0.77\n",
      "loss at batch 400: 0.84\n",
      "Epoch 8\n",
      "loss at batch 0: 0.74\n",
      "loss at batch 100: 0.76\n",
      "loss at batch 200: 0.66\n",
      "loss at batch 300: 0.72\n",
      "loss at batch 400: 0.79\n",
      "Epoch 9\n",
      "loss at batch 0: 0.69\n",
      "loss at batch 100: 0.71\n",
      "loss at batch 200: 0.61\n",
      "loss at batch 300: 0.67\n",
      "loss at batch 400: 0.75\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255\n",
    "\n",
    "fit(model, train_images, train_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.82\n"
     ]
    }
   ],
   "source": [
    "predictions = model(test_images)\n",
    "predictions = predictions.numpy()\n",
    "predicted_labels = np.argmax(predictions, axis=1)\n",
    "matches = predicted_labels == test_labels\n",
    "print(f\"accuracy: {matches.mean():.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Summary"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "chapter02_mathematical-building-blocks.i",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
